{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "zaksum eval.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HY3btgJWEqBP",
        "colab_type": "text"
      },
      "source": [
        "### Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tdcKjPVpEjk8",
        "colab_type": "code",
        "outputId": "78a77680-e219-411e-82d2-c7d548a266ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "#https://stackoverflow.com/questions/47744131/colaboratory-can-i-access-to-my-google-drive-folder-and-file\n",
        "\n",
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()\n",
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}\n",
        "\n",
        "!mkdir -p drive\n",
        "!google-drive-ocamlfuse drive"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "E: Package 'python-software-properties' has no installation candidate\n",
            "Selecting previously unselected package google-drive-ocamlfuse.\n",
            "(Reading database ... 130911 files and directories currently installed.)\n",
            "Preparing to unpack .../google-drive-ocamlfuse_0.7.3-0ubuntu3~ubuntu18.04.1_amd64.deb ...\n",
            "Unpacking google-drive-ocamlfuse (0.7.3-0ubuntu3~ubuntu18.04.1) ...\n",
            "Setting up google-drive-ocamlfuse (0.7.3-0ubuntu3~ubuntu18.04.1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZlHlsJAE75n",
        "colab_type": "text"
      },
      "source": [
        "### Parser"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GBUTmucLFHy4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "article=[]\n",
        "reference=[]\n",
        "summary =[]\n",
        "\n",
        "import xml.etree.ElementTree\n",
        "e = xml.etree.ElementTree.parse(\"drive/Colab Notebooks/Model 5 RL/Policy Gradient/result_policy_gradient_31_5_2019_9_37am.xml\").getroot()\n",
        "\n",
        "for atype in e.findall('example/article'):\n",
        "    article.append(atype.text)\n",
        "    \n",
        "for atype in e.findall('example/reference'):\n",
        "     reference.append(atype.text)\n",
        "    \n",
        "for atype in e.findall('example/summary'):\n",
        "     summary.append(atype.text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g6lMM3PCWZYh",
        "colab_type": "code",
        "outputId": "093dcb85-4426-4470-a01a-ee9490de25d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(summary)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJinxjb1Gwxg",
        "colab_type": "text"
      },
      "source": [
        "### zaksum"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a23wJfc6Gzg4",
        "colab_type": "code",
        "outputId": "72949397-c6f7-42e0-a1f2-63fc6f7c8ecb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        }
      },
      "source": [
        "!pip install sumeval\n",
        "!python -m spacy download en"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sumeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f1/cf/51bcf1d3b48003f7fd0cc6a7c89ef39a252c08acab842143709b5c679ea3/sumeval-0.1.8.tar.gz (44kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 1.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: plac>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from sumeval) (0.9.6)\n",
            "Collecting sacrebleu>=1.3.2 (from sumeval)\n",
            "  Downloading https://files.pythonhosted.org/packages/16/54/165b8d208788b99546032b0053b7c0fa61fa7bf2b55237d78a895168c24e/sacrebleu-1.3.4.tar.gz\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from sacrebleu>=1.3.2->sumeval) (3.6.6)\n",
            "Building wheels for collected packages: sumeval, sacrebleu\n",
            "  Building wheel for sumeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /root/.cache/pip/wheels/6a/f8/76/0b7fc4ff1ed9b973a3edb4ec38f6aa5309b7725d500d3a0d31\n",
            "  Building wheel for sacrebleu (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /root/.cache/pip/wheels/5a/80/a8/e149303db81e58e7dbb29ee7943c43da9861872a5ea550256c\n",
            "Successfully built sumeval sacrebleu\n",
            "Installing collected packages: sacrebleu, sumeval\n",
            "Successfully installed sacrebleu-1.3.4 sumeval-0.1.8\n",
            "Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n",
            "\n",
            "\u001b[93m    Linking successful\u001b[0m\n",
            "    /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n",
            "    /usr/local/lib/python3.6/dist-packages/spacy/data/en\n",
            "\n",
            "    You can now load the model via spacy.load('en')\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7gN0BNbaG35g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://github.com/chakki-works/sumeval\n",
        "#https://github.com/Tian312/awesome-text-summarization\n",
        "\n",
        "from sumeval.metrics.rouge import RougeCalculator\n",
        "from sumeval.metrics.bleu import BLEUCalculator\n",
        "\n",
        "def eval_rouges(refrence_summary,model_summary):\n",
        "    rouge = RougeCalculator(stopwords=True, lang=\"en\")\n",
        "\n",
        "    rouge_1 = rouge.rouge_n(\n",
        "                summary=model_summary,\n",
        "                references=refrence_summary,\n",
        "                n=1)\n",
        "\n",
        "    rouge_2 = rouge.rouge_n(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary],\n",
        "                n=2)\n",
        "    \n",
        "    rouge_l = rouge.rouge_l(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary])\n",
        "    \n",
        "    # You need spaCy to calculate ROUGE-BE\n",
        "    \n",
        "    rouge_be = rouge.rouge_be(\n",
        "                summary=model_summary,\n",
        "                references=[refrence_summary])\n",
        "\n",
        "    bleu = BLEUCalculator()\n",
        "    bleu_score = bleu.bleu( summary=model_summary,\n",
        "                        references=[refrence_summary])\n",
        "    \n",
        "    return rouge_1, rouge_2,rouge_l,rouge_be,bleu_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i6atapqsG52A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://pymotw.com/2/xml/etree/ElementTree/create.html\n",
        "\n",
        "bleu_arr = []\n",
        "rouge_1_arr  = []\n",
        "rouge_2_arr  = []\n",
        "rouge_L_arr  = []\n",
        "rouge_be_arr = []\n",
        "\n",
        "from xml.etree import ElementTree\n",
        "from xml.dom import minidom\n",
        "from functools import reduce\n",
        "\n",
        "def prettify(elem):\n",
        "    \"\"\"Return a pretty-printed XML string for the Element.\n",
        "    \"\"\"\n",
        "    rough_string = ElementTree.tostring(elem, 'utf-8')\n",
        "    reparsed = minidom.parseString(rough_string)\n",
        "    return reparsed.toprettyxml(indent=\"  \")\n",
        "  \n",
        "from xml.etree.ElementTree import Element, SubElement, Comment\n",
        "\n",
        "top = Element('ZakSum')\n",
        "\n",
        "def zaksum(article,reference,summary_array,default_path):\n",
        "  comment = Comment('Generated by Amr Zaki')\n",
        "  top.append(comment)\n",
        "\n",
        "  i=0\n",
        "  for summ in summary_array:\n",
        "    example = SubElement(top, 'example')\n",
        "    article_element   = SubElement(example, 'article')\n",
        "    article_element.text = article[i]\n",
        "\n",
        "    reference_element = SubElement(example, 'reference')\n",
        "    reference_element.text = reference[i]\n",
        "\n",
        "    summary_element   = SubElement(example, 'summary')\n",
        "    summary_element.text = summ\n",
        "\n",
        "    rouge_1, rouge_2,rouge_L,rouge_be,bleu_score = eval_rouges(reference[i],summ )\n",
        "\n",
        "    eval_element = SubElement(example, 'eval')\n",
        "    bleu_score_element = SubElement(eval_element,'BLEU', {'score':str(bleu_score)})\n",
        "    ROUGE_1_element  = SubElement(eval_element, 'ROUGE_1' , {'score':str(rouge_1)})\n",
        "    ROUGE_2_element  = SubElement(eval_element, 'ROUGE_2' , {'score':str(rouge_2)})\n",
        "    ROUGE_L_element  = SubElement(eval_element, 'ROUGE_l' , {'score':str(rouge_L)})\n",
        "    ROUGE_be_element  = SubElement(eval_element,'ROUGE_be', {'score':str(rouge_be)})\n",
        "\n",
        "    bleu_arr.append(bleu_score) \n",
        "    rouge_1_arr.append(rouge_1) \n",
        "    rouge_2_arr.append(rouge_2) \n",
        "    rouge_L_arr.append(rouge_L) \n",
        "    rouge_be_arr.append(rouge_be) \n",
        "\n",
        "    i+=1\n",
        "\n",
        "  top.set('bleu', str(reduce(lambda x, y: x + y,  bleu_arr) / len(bleu_arr)))\n",
        "  top.set('rouge_1', str(reduce(lambda x, y: x + y,  rouge_1_arr) / len(rouge_1_arr)))\n",
        "  top.set('rouge_2', str(reduce(lambda x, y: x + y,  rouge_2_arr) / len(rouge_2_arr)))\n",
        "  top.set('rouge_L', str(reduce(lambda x, y: x + y,  rouge_L_arr) / len(rouge_L_arr)))\n",
        "  top.set('rouge_be', str(reduce(lambda x, y: x + y, rouge_be_arr) / len(rouge_be_arr)))\n",
        "\n",
        "  with open(default_path, \"w\") as f:\n",
        "    print(prettify(top), file=f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O7cG7_wZHdOf",
        "colab_type": "text"
      },
      "source": [
        "#Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCMS9Bb_HQH4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "zaksum(article,reference,summary,\"drive/Colab Notebooks/Model 5 RL/Policy Gradient/result_policy.xml\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mjExkxplhbaU",
        "colab_type": "code",
        "outputId": "3d8e89a6-5622-4f98-9145-bec4e9ea9c63",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "reference"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\"mentally ill inmates in miami are housed on the `` forgotten floor '' judge steven __leifman__ says most are there as a result of `` avoidable felonies '' while cnn tours facility , patient shouts : `` i am the son of the president '' __leifman__ says the system is unjust and he 's fighting for change .\",\n",
              " \"harry potter star daniel radcliffe gets # 20m fortune as he turns 18 monday . young actor says he has no plans to __fritter__ his cash away . radcliffe 's earnings from first five potter films have been held in trust fund .\",\n",
              " \"new : `` i thought i was going to die , '' driver says . man says pickup truck was folded in half ; he just has cut on face . driver : `` i probably had a 30 - , __35-foot__ free fall '' minnesota bridge collapsed during rush hour wednesday .\",\n",
              " \"parents beam with pride , ca n't stop from smiling from outpouring of support . mom : `` i was so happy i did n't know what to do '' burn center in u.s. has offered to provide treatment for reconstructive surgeries . dad says , `` anything for youssif ''\",\n",
              " \"five small __polyps__ found during procedure ; `` none worrisome , '' spokesman says . president !!__reclaims__!! powers transferred to vice president . bush undergoes routine colonoscopy at camp david .\",\n",
              " \"new : nfl chief , atlanta falcons owner critical of michael vick 's conduct . nfl suspends falcons quarterback indefinitely without pay . vick admits funding dogfighting operation but says he did not gamble . vick due in federal court monday ; future in nfl remains uncertain .\",\n",
              " \"aid workers : violence , increased cost of living drive women to prostitution . group is working to raise awareness of the problem with iraq 's political leaders . two iraqi mothers tell cnn they turned to prostitution to help feed their children . `` everything is for the children , '' one woman says .\",\n",
              " \"president bush says tony snow `` will battle cancer and win '' job of press secretary `` has been a dream for me , '' snow says snow leaving on september 14 , will be succeeded by dana perino .\",\n",
              " \"president bush to address the veterans of foreign wars on wednesday . bush to say that withdrawing from vietnam emboldened today 's terrorists . speech will be latest white house attempt to try to !!__reframe__!! the debate over iraq .\",\n",
              " \"new : president bush says he and first lady are deeply saddened by the tragedy . mine safety and health administration chief : we 've run out of options . the six men have been trapped underground since august 6 . seven bore holes drilled into the mountain have found no signs of life .\"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XUdXqYHKhNhm",
        "colab_type": "code",
        "outputId": "9ec95fc1-07b6-4bc6-ecfc-2408ecdbda80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "summary"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\"soledad , soledad o'brien takes users inside a jail in covering before trial . he says face drug charges in miami are mentally ill . the ninth . he is well known in miami before trial ill . [unk]\",\n",
              " 'daniel potter says he has no plans to fritter his cash # 20 million . he insists the money of the world five the world , the young actor he has not been able to be one to touch . [unk]',\n",
              " '`` the whole bridge from one side of the mississippi to the other just completely gave . `` probably was churning and could move around to the scene . he he had could no way of getting to them . [unk]',\n",
              " \"5-year-old youssif held his sister 's sister 's hand . he 's parents . he could n't stop in a request . he says he was so happy to held to help burn victims . [unk]\",\n",
              " \"doctors removed five small polyps from president bush 's colon on saturday , and `` none appeared worrisome , and will resume his activities at camp david colon . the polyps were expected in two to three days . [unk]\",\n",
              " 'nfl star michael vick is set to appear to appear in court monday . a judge will have the final say on a plea deal . the national football league vick in a plea . [unk]',\n",
              " 'suha , 37 , is driven to sell to sell food . she says her husband she is cleaning to sell a day . she was cleaning houses , is cleaning houses when she says . [unk]',\n",
              " \"white house press secretary tony snow will step down from his post on september 14 . `` will `` will sadly accept '' he will `` sadly accept for cancer 's resignation . `` he he 'll be a a solid contributor . [unk]\",\n",
              " 'president bush will tell will try to put a twist on iraq of the war summit in canada city . bush bush will say will tell members of the veterans of foreign wars in canada . [unk]',\n",
              " 'isaac is `` no remaining hope in a utah for miners . `` over the past 25 in price , the mine . `` the failed and prayers . `` are no sign of the six miners . [unk]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8qRwSyz0hN0c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}